package test

import cn.edu.hfut.dmic.webcollector.model._
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler
import command.CommandExec
import command.CommandPojo.CommandData
import implicits.Implicits._
import utils.Utils

import java.io.File
import java.net.URL
import scala.collection.JavaConverters._
import scala.sys.process._

/** Created by yz on 19/7/2021
  */
object GenomicCrawler {

  val parentDir = new File("E:\\mtd_database\\test")
  val outDir = new File(parentDir, "genomic")

  class Crawler(crawlPath: String, autoParse: Boolean) extends BreadthCrawler(crawlPath, autoParse) {
    val startTime = System.currentTimeMillis()
    val maps = new File(parentDir, "prokaryotes.csv").csvLines.lineSeqMap

    val emptyBioSamples = maps
      .filter { map =>
        val id = map("BioSample")
        val dir = new File(outDir, id)
        dir.myListFiles.exists(_.getName.endsWith(".genomic.fna.gz"))
      }
      .filter { map =>
        val id = map("BioSample")
        val dir = new File(outDir, id)
        val file = dir.myListFiles.find(_.getName.endsWith(".genomic.fna.gz")).get
        file.size == 0
      }
      .map(_("BioSample"))
    println(emptyBioSamples.size)
    println(emptyBioSamples)

    val linkIdMap = maps
      .filter { map =>
        val id = map("BioSample")
        val dir = new File(outDir, id)
        val fileOp = dir.myListFiles.find(_.getName.endsWith(".genomic.fna.gz"))
        val refLink = map("RefSeq FTP")
        val genLink = map("GenBank FTP")
        (fileOp.isEmpty) || (fileOp.nonEmpty && fileOp.get.size == 0) || {
          (!dir.myListFiles.exists(_.getName.endsWith(".genomic.fna"))) && {
            val gzFile = dir.myListFiles.find(_.getName.endsWith(".fna.gz")).get
            val command =
              s"""
                 |gunzip -k ${gzFile.unixPath}
                 |""".stripMargin
            val commandExec = CommandExec().exec { b =>
              CommandData(dir, command)
            }
            if (!commandExec.isSuccess) {
              println(id, commandExec.errorInfo)
            }
            (!commandExec.isSuccess)
          }
        }
      }
      .map { map =>
        val refLink = map("RefSeq FTP")
        val genLink = map("GenBank FTP")
        val ftpLink = if (refLink.nonBlank) {
          refLink
        } else genLink
        val link = s"${ftpLink.replaceAll("^ftp", "https")}/"
        (link, map("BioSample"))
      }
      .filter { case (k, v) =>
        k.nonBlank && (k != "/")
      }
      .toSeqMap

    println(linkIdMap.size, linkIdMap)

    val links = linkIdMap.keyList

    links.foreach { x =>
      this.addSeed(s"${x}")
    }
    var i = 0

    override def visit(page: Page, next: CrawlDatums): Unit = {
      val url = page.url()
      try {
        val link = page
          .links()
          .asScala
          .toList
          .filter { x =>
            val endStr = s"${url.mySplit("/").last}_genomic.fna.gz"
            x.endsWith(endStr)
          }
          .head
        val id = linkIdMap(url)
        val eachOutDir = new File(outDir, id).createDirectoryWhenNoExist

        (new URL(link).#>(new File(eachOutDir, s"${id}.genomic.fna.gz"))).!!
        i += 1
        println(i, linkIdMap.size, Utils.getTime(startTime))
      } catch {
        case e: Exception =>
          e.printStackTrace()
          println(url)
      }
    }
  }

  def main(args: Array[String]): Unit = {

    val crawler = new Crawler("crawl", false)
    crawler.setThreads(10)
    crawler.start(1)
  }

}
